# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import json
import codecs
import os


class NovelgetPipeline:

    def __init__(self):
        self.content_list = []

    def open_spider(self, spider):
        self.file = codecs.open('res/done.txt', 'w','utf-8')

    def process_item(self, item, spider):
        result = {}
        dir = 'res/'
        filename = dir + 'list' + item['index'] + '.json'

        if not os.path.exists(dir):
            os.makedirs(dir)

        # file = codecs.open(filename, 'w', 'utf-8')
        print(item['index'])
        # del item['index']

        self.content_list.append(item)
        for key in item:
            if key!='index':
                print(item[key]['name'])
                print(item[key]['description'])
        # content = json.dumps(item, ensure_ascii=False) + "\n"
        # file.write(content)
        return item

    def close_spider(self, spider):
        list_sorted = sorted(self.content_list, key=lambda x:x['index'])
        for item in list_sorted:
            self.file.write("----------------%s--------------------------------\n" % (item['index']))
            # self.file.write(''.join(item['content']).replace('\xa0', '') + "\n")
            content = json.dumps(item, ensure_ascii=False) + "\n"
            self.file.write(content)
        self.file.close()

class NovelContextPipeline:
    def __init__(self):
        self.content_list = []

    def open_spider(self, spider):
        self.file = codecs.open('res/done.txt', 'a+','utf-8')

    def process_item(self, item, spider):
        self.content_list.append(item)
        return item

    def close_spider(self, spider):
        list_sorted = sorted(self.content_list, key=lambda x:x['index'])
        for item in list_sorted:
            print(item['title'])
            self.file.write("----------------%s--------------------------------\n" % (item['title']))
            content = item['context']+"\n"
            self.file.write(content)
        self.file.close()